import json
import re
import scrapy
from SpiderDemo9.items import JingDoCommentItem


class JingdocommentspiderSpider(scrapy.Spider):
    name = 'JingDoCommentSpider'

    # allowed_domains = ['jd.com']
    # 使用请求头访问，伪装成浏览器
    header = {
        ":authority": "club.jd.com",
        ":method": "GET",
        ":path": "/comment/productPageComments.action?callback=fetchJSON_comment98&productId=37245978364&score=0&sortType=5&page=0&pageSize=10&isShadowSku=0&fold=1",
        ":scheme": "https",
        "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
        "accept-language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7",
        "referer": "https://item.jd.com/",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36"
    }

    def start_requests(self):
        base_url = 'https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId=100016264756&score=0&sortType=5&page={}&pageSize=10&isShadowSku=0&fold=1'
        for i in range(10):
            url = base_url.format(i)
            yield scrapy.Request(url=url, headers=self.header, callback=self.parse)

    def parse(self, response):
        print(response.text)
        # 使用正则表达式将fetchJSON_comment98({......}); 的json提取出来
        r = re.compile(r"[(](.*)[)]", re.S)
        rlist = r.findall(response.text)
        # print(rlist[0])
        # 使用 json模块 加载 json字符串
        jsonObject = json.loads(rlist[0])
        comments = jsonObject['comments']
        for comment in comments:
            jdCommentItem = JingDoCommentItem()
            jdCommentItem["user_id"] = comment.get("id", None)
            jdCommentItem["username"] = comment.get("nickname", None)
            jdCommentItem["comment"] = comment.get("content", None)
            jdCommentItem["score"] = comment.get("score", None)
            jdCommentItem["create_time"] = comment.get("creationTime", None)
            # 将从response中提取到的Item 发送到PipeLine中处理
            yield jdCommentItem
